#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-10-25 10:04:36
# Project: m_autohome_com_cn_comment2

import re
import json
import html
import time
import random
import string
import requests

from pyspider.libs.base_handler import *
from pyquery import PyQuery as pq

from src.mysqldb import SQL
from src.headers_switch import HeadersSelector
from src.utils import download_img
from src.utils import md5
from src.utils import oss
from src.utils import get_proxy


class Handler(BaseHandler):
    crawl_config = {
        'itag': 'v0.1.4',
        'proxy': '',
    }

    cookies = ''

    def get_cookies(self):
        cookie = {}
        return cookie
        for line in self.cookies.split(';'):
            key,value = line.split('=', 1) #1代表只分一次，得到两个数据
            cookie[key.strip()] =value.strip()
        return cookie

    @every(minutes=30)
    def on_start(self):
        self.crawl_config['proxy'] = get_proxy()
        header_slt = HeadersSelector()
        header = header_slt.select_header()  # 获取一个新的 header

        header['Host'] = 'k.m.autohome.com.cn'

        cli = [
            (33211, ["https://k.m.autohome.com.cn/detail/view_01ckdhv8qm68rk4dsp64v00000.html#pvareaid=2147101"]),
            (33259, ["https://k.m.autohome.com.cn/detail/view_01ckd2b11r68rk4dhr68tg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01c9dxypg164wkac9n6cvg0000.html#pvareaid=2147101"]),(33304, ["https://k.m.autohome.com.cn/detail/view_01cqps71bf68s30dsn6cvg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01c4k40g7364w3ge1m74rg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cp1ynhxd68rkce9q6wt00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cf7v68m568r3adsq64t00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01c8ddenc764wk6dsg6ms00000.html#pvareaid=2147101"]),(33568, ["https://k.m.autohome.com.cn/detail/view_01ct2wxknf68s38chp68s00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01ccn4mbvh68r32csk70tg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cpn3yb6f68rkge9m60sg0000.html#pvareaid=2147101"]),(33655, ["https://k.m.autohome.com.cn/detail/view_01ckqxwee568rk6d1p64r00000.html#pvareaid=2147101"]),(33853, ["https://k.m.autohome.com.cn/detail/view_01cj43n35x68rk0d1s68v00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cq1bxbze68rkjd1m6mtg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cj1dc8g568rk0csn6mrg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01ch4nav1g68r3jc1r6wr00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cfce2k2w68r3ae9s6rt00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cs9dytqg68s36c1m6msg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cnvb3kx068rkcd9s74v00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cfde7dqj68r3cc1p70rg0000.html#pvareaid=2147101"]),(33896, ["https://k.m.autohome.com.cn/detail/view_01cb47hawb64wkedsq6mt00000.html#pvareaid=2147101"]),(33998, ["https://k.m.autohome.com.cn/detail/view_01cka0fwnb68rk4d1p6wtg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01chj1zdeq68r3jdhn6rw00000.html#pvareaid=2147101"]),(34203, ["https://k.m.autohome.com.cn/detail/view_01csxtxz2a68s36e9r64tg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cpqa3an168rkjc1j6mw00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cqwwdssn68s32c1k6rvg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cfpdgw2k68r3cd1m6cw00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cp6tam6368rkechr6mv00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01ch2r7at668r3ge9s6msg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cjy001m768rk2dsh68tg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cshfmqrv68s36d1j70w00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cns06h6568rkcd1r6mvg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cnd904d568rkadss74s00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cjykaxpr68rk2dsm6ct00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01ct02vejy68s38c1s6gtg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cpa5v9fr68rked1s74r00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cngqrjpx68rkcc1h60vg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cmw020g368rkac1k64wg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cmgv5p3b68rk8d9g70r00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01ck7mtghj68rk4chs68sg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01ct5at6pn68s38d1g6gs00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cs62fnkf68s34e9g68v00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cp7z8apz68rked1g70sg0000.html#pvareaid=2147101"]),(34349, ["https://k.m.autohome.com.cn/detail/view_01csb9bjky68s36c9h68w00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cmnset8n68rk8dsj6gr00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01crtgvnw468s34dhh6rv00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cqv11acf68s30e9p6mw00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cnzqeqxc68rkce1m6mv00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cmw8egch68rkac1m70rg0000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cp4sz8jh68rkec9p74t00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cp2gy45g68rkec1m6csg0000.html#pvareaid=2147101"]),(34407, ["https://k.m.autohome.com.cn/detail/view_01crwnf4p468s34dhp6ct00000.html#pvareaid=2147101"]),(34567, ["https://k.m.autohome.com.cn/detail/view_01cpsd5c7x68rkjc1s70s00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cpkyw8g368rkge1p74vg0000.html#pvareaid=2147101"]),(34600, ["https://k.m.autohome.com.cn/detail/view_01cnrph3e168rkcd1p70s00000.html#pvareaid=2147101", "https://k.m.autohome.com.cn/detail/view_01cq3jn30c68rkjd9m6msg0000.html#pvareaid=2147101"]),(34758, ["https://k.m.autohome.com.cn/detail/view_01cms0h74b68rk8e1p70s00000.html#pvareaid=2147101"]),]
        js_script = """
            function() {
                setTimeout(function() {
                }, 3000);
            }
        """
        for (gid, cli2) in cli:
            save = {
                'gid': gid,
                'tagid': '0',
            }
            for url in cli2:
                yield self.crawl(url, callback=self.detail_page, save=save, headers=header, fetch_type='js', js_script=js_script)


    @config(priority=2)
    def detail_page(self, response):
        self.crawl_config['proxy'] = get_proxy()
        header_slt = HeadersSelector()
        header = header_slt.select_header()  # 获取一个新的 header

        doc = response.doc
        save = response.save
        gid = save.get('gid', '0')
        tagid = save.get('tagid', '0')

        comment = {}
        comment['url'] = response.url
        comment['goods_id'] = gid
        comment['goods_name'] = doc('.cartype a').text()
        comment['title'] = doc('h1').text()
        comment['category'] = ''
        comment['tagid'] = tagid
        comment['author'] = doc('.wom-details .info .name span').text()
        comment['pubtime'] = doc('.wom-details .info time').text()
        comment['dealer'] = doc('.site').text()
        comment['buy_date'] = doc('.date span').text()
        items = list(doc('.unit-gather .item').items())
        if not items:
            items = list(doc('.edition-base .unit-gather .item').items())
        #print(items, comment)
        comment['naked_car_price'] = items[0].text()
        comment['oil_wear'] = items[1].text()
        comment['current_drive'] = items[2].text()
        comment['content'] = html.escape(doc('.matter').html())
        return comment

    def on_result(self,result):
        print(result)
        if not result or not result.get('goods_id', ''):
            return
        sql = SQL()
        header = result.get('header', None)
        if header:
            result.pop('header')
        # print(result)
        sql.replace('jjb_goods_car_comment', result)